{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request as ur\n",
    "import tool\n",
    "import re\n",
    "import json\n",
    "import user_agent\n",
    "import lxml.etree as le\n",
    "import pymongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://search.51job.com/list/090200,000000,0000,00,9,99,python,2,2.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='\n",
    "req=ur.Request(\n",
    "    url=url,\n",
    "    headers={\n",
    "        'User-Agent':user_agent.get_user_agent_pc()\n",
    "    }\n",
    ")\n",
    "content =ur.urlopen(req).read().decode('gbk','ignore')\n",
    "# print(content)\n",
    "\n",
    "# with open('51job.html','w',encoding='gbk') as f:\n",
    "#     f.write(content)\n",
    "    \n",
    "ret=re.findall('window.__SEARCH_RESULT__ = (.*?)</script>',content)\n",
    "# print(ret)\n",
    "#json转换位字典\n",
    "data=json.loads(ret[0])\n",
    "# print(data)\n",
    "# resoults=(data['engine_search_result'])\n",
    "# for resoult in resoults:\n",
    "#     print(resoult)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse1(keyword,page):\n",
    "    url='https://search.51job.com/list/090200,000000,0000,00,9,99,{keyword},2,{page}.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='.format(\n",
    "        keyword=keyword,\n",
    "        page=page\n",
    "    )\n",
    "    req=ur.Request(\n",
    "        url=url,\n",
    "        headers={\n",
    "            'User-Agent':user_agent.get_user_agent_pc()\n",
    "        }\n",
    "    )\n",
    "    content =ur.urlopen(req).read().decode('gbk','ignore')\n",
    "    data=json.loads(\n",
    "        re.findall('window.__SEARCH_RESULT__ = (.*?)</script>',content)[0]\n",
    "        )\n",
    "    resoults=(data['engine_search_result'])\n",
    "    return resoults\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if __name__ == '__main__':\n",
    "#     print(parse1(keyword='python',page=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    #content=ur.urlopen('')\n",
    "    content1=ur.urlopen('https://jobs.51job.com/chongqing/126010937.html?s=01&t=0').read()\n",
    "    with open('51job2.html','wb') as f :\n",
    "        f.write(content1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "content2=ur.urlopen(url='https://jobs.51job.com/chongqing/126010937.html?s=01&t=0').read()\n",
    "contentx=le.HTML(content2)\n",
    "resoults=contentx.xpath('//div[@class=\"tCompany_main\"]/div[@class=\"tBorderTop_box\"][1]//text()')\n",
    "\n",
    "ret_data=[\n",
    "    ret.strip() for ret in resoults\n",
    "]\n",
    "ret_data1=''.join([\n",
    "    ret.strip() for ret in resoults\n",
    "])\n",
    "\n",
    "# print(ret_data)\n",
    "# print(ret_data1)\n",
    "parse2_job_detail=tool.xpath_union(contentx=contentx,path='//div[@class=\"tCompany_main\"]/div[@class=\"tBorderTop_box\"][1]//text()',split='\\n',default=None)\n",
    "parse2_job_contact=tool.xpath_union(contentx=contentx,path='//div[@class=\"tCompany_main\"]/div[@class=\"tBorderTop_box\"][2]//text()',split='\\n',default=None)\n",
    "parse2_job_company=tool.xpath_union(contentx=contentx,path='//div[@class=\"tCompany_main\"]/div[@class=\"tBorderTop_box\"][3]//text()',split='\\n',default=None)\n",
    "# print(parse2_job_detail)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse2(url):\n",
    "    req=ur.Request(\n",
    "        url=url,\n",
    "        headers={\n",
    "            'User-Agent':user_agent.get_user_agent_pc()\n",
    "        }\n",
    "    )\n",
    "    content2 =ur.urlopen(req).read()\n",
    "    contentx=le.HTML(content2)\n",
    "    \n",
    "    parse2_job_detail=tool.xpath_union(contentx=contentx,path='//div[@class=\"tCompany_main\"]/div[@class=\"tBorderTop_box\"][1]//text()',split='',default=None)\n",
    "    parse2_job_contact=tool.xpath_union(contentx=contentx,path='//div[@class=\"tCompany_main\"]/div[@class=\"tBorderTop_box\"][2]//text()',split='',default=None)\n",
    "    parse2_job_company=tool.xpath_union(contentx=contentx,path='//div[@class=\"tCompany_main\"]/div[@class=\"tBorderTop_box\"][3]//text()',split='',default=None)\n",
    "    return dict(\n",
    "        parse2_job_detail=parse2_job_detail,\n",
    "        parse2_job_contact=parse2_job_contact,\n",
    "        parse2_job_company=parse2_job_company,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spider51(keyword,c,start_page=1,end_page=100):\n",
    "    for page in range(start_page,end_page+1):\n",
    "        parse1_datas=parse1(keyword=keyword,page=page)\n",
    "        for parse1_data in parse1_datas:\n",
    "            job_href=parse1_data['job_href']\n",
    "            parse2_data=parse2(url=job_href)\n",
    "            parse1_data['parse2_job_detail']=parse2_data['parse2_job_detail']\n",
    "            parse1_data['parse2_job_contact']=parse2_data['parse2_job_contact']\n",
    "            parse1_data['parse2_job_company']=parse2_data['parse2_job_company']\n",
    "            #print(parse1_data)\n",
    "            #保存数据\n",
    "            c.insert_one(parse1_data)\n",
    "\n",
    "# if __name__ == '__main__':\n",
    "#     #print(parse2(url='https://jobs.51job.com/chongqing/126010937.html?s=01&t=0'))\n",
    "#     parse1_datas=parse1(keyword='java',page=1)\n",
    "#     for parse1_data in parse1_datas:\n",
    "#         job_href=parse1_data['job_href']\n",
    "#         parse2_data=parse2(url=job_href)\n",
    "#         parse1_data['parse2_job_detail']=parse2_data['parse2_job_detail']\n",
    "#         parse1_data['parse2_job_contact']=parse2_data['parse2_job_contact']\n",
    "#         parse1_data['parse2_job_company']=parse2_data['parse2_job_company']\n",
    "#         print(parse1_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    client=pymongo.MongoClient()\n",
    "    db=client.get_database('db01')#db=client.db01\n",
    "    c=db.get_collection('51job')#c=db.51job\n",
    "    spider51(keyword='python',c=c,start_page=1,end_page=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
